Benchmark de modelo

Cargando paquetes

Rutas

raw_data <- here("data", "raw")
interim_data <- here("data", "interim")
processed_data <- here("data", "processed")

Leyendo bases

base <- readRDS(file=paste0(processed_data,"/base_final.Rds"))

base <- base %>%
  mutate(objetivo = factor(if_else(objetivo==0, "bueno", "malo")))

head(base)

Tarea de aprendizaje

Definicion

task_tarj = TaskClassif$new(id = "tarjetas", backend = base, target = "objetivo")
print(task_tarj)

<TaskClassif:tarjetas> (14500 x 9)
* Target: objetivo
* Properties: twoclass
* Features (8):
  - dbl (4): coeficiente_solvencia, edad, limite_tarjeta_credito, saldo_tarjeta
  - fct (4): estado_civil, genero, profesion, tipo_ingresos

#autoplot(task_tarj$select(c("saldo_tarjeta", "coeficiente_solvencia", "edad", "limite_tarjeta_credito")), type = "pairs")
#task_tarj = TaskClassif$new(id = "tarjetas", backend = base, target = "objetivo")

Modelos

Lista de modelos

lrn_rpart = lrn("classif.rpart", predict_type = "prob")
lrn_glmnet = lrn("classif.glmnet", predict_type = "prob")
lrn_knn = lrn("classif.kknn", predict_type = "prob")
lrn_lda = lrn("classif.lda", predict_type = "prob")
lrn_nnet = lrn("classif.nnet", predict_type = "prob")
lrn_rf = lrn("classif.ranger", predict_type = "prob")
lrn_svm = lrn("classif.svm", predict_type = "prob")
lrn_xgboost = lrn("classif.xgboost", predict_type = "prob")

Separacion de training y testing

set.seed(34678)
train_set = sample(task_tarj$nrow, 0.8 * task_tarj$nrow)
test_set = setdiff(seq_len(task_tarj$nrow), train_set)

Benchmark

Prepracion de datos para el benchmark

th = prop.table(table(base$objetivo))[2]
impute_fcts <- po("imputemode", affect_columns = selector_type("factor"))
impute_nums <- po("imputehist", affect_columns = selector_type("numeric"))
encode <- po("encode", affect_columns = selector_type("factor"))
threshold <- po("threshold", param_vals = list(thresholds = th))

pre_procesamiento <- impute_fcts %>>% 
  impute_nums %>>% 
  encode

lrn_rpart$predict_type = "prob"

Error in lrn_rpart$predict_type = "prob" : object 'lrn_rpart' not found

Defincion del benchmark

resamplings = rsmp("cv", folds = 10)

design = benchmark_grid(task_tarj, learners, resamplings)

Ejecucion

future::plan("multiprocess")

[ONE-TIME WARNING] Forked processing ('multicore') is disabled in future (>= 1.13.0) when running R from RStudio, because it is considered unstable. Because of this, plan("multicore") will fall back to plan("sequential"), and plan("multiprocess") will fall back to plan("multisession") - not plan("multicore") as in the past. For more details, how to control forked processing or not, and how to silence this warning in future R sessions, see ?future::supportsMulticore

bmr = benchmark(design)

INFO  [23:20:42.885] Benchmark with 80 resampling iterations 
INFO  [23:20:49.124] Applying learner 'XGBoost' on task 'tarjetas' (iter 5/10) 
INFO  [23:20:55.533] Applying learner 'SVM' on task 'tarjetas' (iter 1/10) 
INFO  [23:30:42.494] Applying learner 'Reg-reg' on task 'tarjetas' (iter 8/10) 
INFO  [23:30:45.129] Applying learner 'LDA' on task 'tarjetas' (iter 4/10) 
INFO  [23:30:47.410] Applying learner 'XGBoost' on task 'tarjetas' (iter 2/10) 
INFO  [23:30:50.448] Applying learner 'LDA' on task 'tarjetas' (iter 9/10) 
INFO  [23:30:53.122] Applying learner 'SVM' on task 'tarjetas' (iter 7/10) 
INFO  [23:34:11.861] Applying learner 'Reg-reg' on task 'tarjetas' (iter 5/10) 
INFO  [23:34:13.381] Applying learner 'K-vecinos' on task 'tarjetas' (iter 9/10) 
INFO  [23:34:22.641] Applying learner 'Árbol' on task 'tarjetas' (iter 6/10) 
INFO  [23:34:24.613] Applying learner 'LDA' on task 'tarjetas' (iter 10/10) 
INFO  [23:34:26.353] Applying learner 'LDA' on task 'tarjetas' (iter 1/10) 
INFO  [23:34:27.749] Applying learner 'SVM' on task 'tarjetas' (iter 5/10) 
INFO  [23:36:55.046] Applying learner 'Red-Neur' on task 'tarjetas' (iter 6/10) 
# weights:  88
initial  value 8864.344041 
iter  10 value 3550.653729
iter  20 value 3525.984748
iter  30 value 3524.750652
iter  40 value 3517.336095
iter  50 value 3504.527029
iter  60 value 3490.270601
iter  70 value 3488.355126
final  value 3488.124393 
converged
INFO  [23:36:57.713] Applying learner 'Red-Neur' on task 'tarjetas' (iter 7/10) 
# weights:  88
initial  value 9808.686149 
final  value 3655.011275 
converged
INFO  [23:36:59.122] Applying learner 'LDA' on task 'tarjetas' (iter 2/10) 
INFO  [23:37:00.530] Applying learner 'Red-Neur' on task 'tarjetas' (iter 9/10) 
# weights:  88
initial  value 18674.516815 
iter  10 value 3562.604417
final  value 3562.604078 
converged
INFO  [23:37:02.284] Applying learner 'RandomForest' on task 'tarjetas' (iter 6/10) 
INFO  [23:37:13.358] Applying learner 'Reg-reg' on task 'tarjetas' (iter 4/10) 
INFO  [23:37:14.932] Applying learner 'LDA' on task 'tarjetas' (iter 3/10) 

variables are collinearvariables are collinearvariables are collinearvariables are collinearvariables are collinearvariables are collinear

INFO  [23:21:07.904] Applying learner 'LDA' on task 'tarjetas' (iter 8/10) 
INFO  [23:21:18.839] Applying learner 'Árbol' on task 'tarjetas' (iter 10/10) 
INFO  [23:21:26.230] Applying learner 'Reg-reg' on task 'tarjetas' (iter 9/10) 
INFO  [23:21:34.822] Applying learner 'Árbol' on task 'tarjetas' (iter 8/10) 
INFO  [23:21:52.375] Applying learner 'K-vecinos' on task 'tarjetas' (iter 10/10) 
INFO  [23:23:28.127] Applying learner 'K-vecinos' on task 'tarjetas' (iter 3/10) 
INFO  [23:24:53.842] Applying learner 'RandomForest' on task 'tarjetas' (iter 7/10) 
Growing trees.. Progress: 78%. Estimated remaining time: 8 seconds.
INFO  [23:26:07.116] Applying learner 'XGBoost' on task 'tarjetas' (iter 6/10) 
INFO  [23:26:29.350] Applying learner 'RandomForest' on task 'tarjetas' (iter 10/10) 
Growing trees.. Progress: 73%. Estimated remaining time: 11 seconds.
INFO  [23:27:39.301] Applying learner 'Red-Neur' on task 'tarjetas' (iter 8/10) 
# weights:  88
initial  value 5082.615183 
final  value 3662.311866 
converged
INFO  [23:27:48.348] Applying learner 'Red-Neur' on task 'tarjetas' (iter 3/10) 
# weights:  88
initial  value 7879.719197 
final  value 3659.879369 
converged
INFO  [23:28:03.441] Applying learner 'Red-Neur' on task 'tarjetas' (iter 10/10) 
# weights:  88
initial  value 9476.761085 
final  value 3663.376169 
converged
INFO  [23:28:20.336] Applying learner 'LDA' on task 'tarjetas' (iter 7/10) 
INFO  [23:28:29.218] Applying learner 'Reg-reg' on task 'tarjetas' (iter 2/10) 
INFO  [23:28:38.864] Applying learner 'K-vecinos' on task 'tarjetas' (iter 4/10) 
INFO  [23:29:21.316] Applying learner 'XGBoost' on task 'tarjetas' (iter 3/10) 
INFO  [23:29:24.120] Applying learner 'Reg-reg' on task 'tarjetas' (iter 1/10) 
INFO  [23:29:26.536] Applying learner 'Reg-reg' on task 'tarjetas' (iter 3/10) 
INFO  [23:29:28.883] Applying learner 'SVM' on task 'tarjetas' (iter 10/10) 
INFO  [23:33:10.737] Applying learner 'LDA' on task 'tarjetas' (iter 6/10) 

variables are collinearvariables are collinearvariables are collinear

INFO  [23:21:29.075] Applying learner 'K-vecinos' on task 'tarjetas' (iter 7/10) 
INFO  [23:23:00.642] Applying learner 'K-vecinos' on task 'tarjetas' (iter 8/10) 
INFO  [23:24:36.405] Applying learner 'Árbol' on task 'tarjetas' (iter 4/10) 
INFO  [23:24:47.095] Applying learner 'Red-Neur' on task 'tarjetas' (iter 4/10) 
# weights:  88
initial  value 8002.400008 
final  value 3606.101478 
converged
INFO  [23:24:55.663] Applying learner 'RandomForest' on task 'tarjetas' (iter 2/10) 
Growing trees.. Progress: 76%. Estimated remaining time: 9 seconds.
INFO  [23:26:10.267] Applying learner 'RandomForest' on task 'tarjetas' (iter 9/10) 
Growing trees.. Progress: 89%. Estimated remaining time: 3 seconds.
INFO  [23:27:28.514] Applying learner 'SVM' on task 'tarjetas' (iter 8/10) 
INFO  [23:32:16.368] Applying learner 'RandomForest' on task 'tarjetas' (iter 1/10) 
INFO  [23:32:34.878] Applying learner 'SVM' on task 'tarjetas' (iter 3/10) 
INFO  [23:35:09.973] Applying learner 'Reg-reg' on task 'tarjetas' (iter 10/10) 
INFO  [23:35:11.826] Applying learner 'RandomForest' on task 'tarjetas' (iter 4/10) 
INFO  [23:35:27.394] Applying learner 'Árbol' on task 'tarjetas' (iter 2/10) 
INFO  [23:35:30.141] Applying learner 'Reg-reg' on task 'tarjetas' (iter 7/10) 
INFO  [23:35:32.224] Applying learner 'XGBoost' on task 'tarjetas' (iter 8/10) 
INFO  [23:35:34.267] Applying learner 'XGBoost' on task 'tarjetas' (iter 9/10) 
INFO  [23:35:36.310] Applying learner 'SVM' on task 'tarjetas' (iter 4/10) 
INFO  [23:37:46.667] Applying learner 'SVM' on task 'tarjetas' (iter 9/10) 
INFO  [23:39:39.158] Applying learner 'Red-Neur' on task 'tarjetas' (iter 2/10) 
# weights:  88
initial  value 8668.465816 
final  value 3647.701370 
converged
INFO  [23:39:40.093] Applying learner 'Árbol' on task 'tarjetas' (iter 5/10) 
INFO  [23:39:41.415] Applying learner 'SVM' on task 'tarjetas' (iter 2/10) 
INFO  [23:21:52.892] Applying learner 'K-vecinos' on task 'tarjetas' (iter 5/10) 
INFO  [23:23:35.269] Applying learner 'Árbol' on task 'tarjetas' (iter 3/10) 
INFO  [23:23:49.976] Applying learner 'XGBoost' on task 'tarjetas' (iter 7/10) 
INFO  [23:24:05.010] Applying learner 'K-vecinos' on task 'tarjetas' (iter 6/10) 
INFO  [23:25:50.915] Applying learner 'RandomForest' on task 'tarjetas' (iter 8/10) 
INFO  [23:27:07.196] Applying learner 'Árbol' on task 'tarjetas' (iter 9/10) 
INFO  [23:27:30.417] Applying learner 'Árbol' on task 'tarjetas' (iter 1/10) 
INFO  [23:27:41.453] Applying learner 'XGBoost' on task 'tarjetas' (iter 1/10) 
INFO  [23:27:52.590] Applying learner 'RandomForest' on task 'tarjetas' (iter 3/10) 
INFO  [23:28:38.794] Applying learner 'Red-Neur' on task 'tarjetas' (iter 5/10) 
# weights:  88
initial  value 12571.518812 
final  value 3650.139041 
converged
INFO  [23:28:44.333] Applying learner 'RandomForest' on task 'tarjetas' (iter 5/10) 
INFO  [23:29:13.585] Applying learner 'XGBoost' on task 'tarjetas' (iter 4/10) 
INFO  [23:29:15.707] Applying learner 'LDA' on task 'tarjetas' (iter 5/10) 
INFO  [23:29:18.341] Applying learner 'XGBoost' on task 'tarjetas' (iter 10/10) 
INFO  [23:29:20.512] Applying learner 'SVM' on task 'tarjetas' (iter 6/10) 
INFO  [23:33:05.999] Applying learner 'Reg-reg' on task 'tarjetas' (iter 6/10) 
INFO  [23:33:08.879] Applying learner 'Árbol' on task 'tarjetas' (iter 7/10) 
INFO  [23:33:11.163] Applying learner 'K-vecinos' on task 'tarjetas' (iter 1/10) 
INFO  [23:33:27.476] Applying learner 'Red-Neur' on task 'tarjetas' (iter 1/10) 
# weights:  88
initial  value 8686.105637 
iter  10 value 3586.420021
iter  10 value 3586.420021
iter  10 value 3586.420021
final  value 3586.420021 
converged
INFO  [23:33:29.287] Applying learner 'K-vecinos' on task 'tarjetas' (iter 2/10) 

variables are collinear

INFO  [23:41:29.683] Finished benchmark 

measures = list(
  msr("classif.acc", id = "acc_train", predict_sets = "train"),
  msr("classif.acc", id = "acc_test"),
  msr("classif.ppv", id = "ppv_train", predict_sets = "train"),
  msr("classif.ppv", id = "ppv_test"),
  msr("classif.tpr", id = "tpr_train", predict_sets = "train"),
  msr("classif.tpr", id = "tpr_test")
)

bmr$aggregate(measures)

NA

tab = bmr$aggregate(measures)

ranks = tab[, .(learner_id, rank_train = rank(-acc_train), rank_test = rank(-acc_test)), by = task_id]
print(ranks)

autoplot(bmr$clone(deep = TRUE), type = "roc")

autoplot(bmr$clone(deep = TRUE), type = "roc")

autoplot(bmr$clone(deep = TRUE), type = "roc")

autoplot(bmr, measure = msr("classif.tpr")) + 
           theme(axis.text.x = element_text(angle = 45, hjust = 1))

NA
NA

autoplot(bmr, measure = msr("classif.ppv")) + 
           theme(axis.text.x = element_text(angle = 45, hjust = 1))

NA
NA

pred_fr = lrn_rf_2$predict(task_tarj, row_ids = test_set)
pred_fr = lrn_rf_2$predict(task_tarj, row_ids = test_set)
pred_rf$confusion

Error: object 'pred_rf' not found

lrn_xgboost_2 = lrn_xgboost

lrn_xgboost_2$train(task_tarj, row_ids = train_set)
pred_xg = lrn_xgboost_2$predict(task_tarj, row_ids = test_set)

pred_xg$confusion

        truth
response bueno malo
   bueno  2679  221
   malo      0    0

lrn_xgboost_2 = lrn_xgboost

Costos

costs = matrix(c(0, 1, 0, -.9), nrow = 2)
dimnames(costs) = list(response = c("bueno", "malo"), truth = c("bueno", "malo"))
print(costs)

        truth
response bueno malo
   bueno     0  0.0
   malo      1 -0.9

table(task_tarj$truth())

bueno  malo 
13334  1166 

(13334 * costs[2, 1] + 1166 * costs[2, 2]) / 14500

[1] 0.8472138

(13334 * costs[1, 1] + 1166 * costs[1, 2]) / 14500

[1] 0

.85*1000*1000

[1] 850000

Modelo sencillo

learner = GraphLearner$new(pre_procesamiento %>>% po(lrn("classif.rpart"))) 
rr = resample(task_tarj, learner, rsmp("cv"))

INFO  [20:29:34.716] Applying learner 'imputemode.imputehist.encode.classif.rpart' on task 'tarjetas' (iter 8/10) 
INFO  [20:29:35.816] Applying learner 'imputemode.imputehist.encode.classif.rpart' on task 'tarjetas' (iter 4/10) 
INFO  [20:29:36.441] Applying learner 'imputemode.imputehist.encode.classif.rpart' on task 'tarjetas' (iter 1/10) 
INFO  [20:29:36.987] Applying learner 'imputemode.imputehist.encode.classif.rpart' on task 'tarjetas' (iter 10/10) 
INFO  [20:29:37.616] Applying learner 'imputemode.imputehist.encode.classif.rpart' on task 'tarjetas' (iter 5/10) 
INFO  [20:29:38.203] Applying learner 'imputemode.imputehist.encode.classif.rpart' on task 'tarjetas' (iter 9/10) 
INFO  [20:29:38.803] Applying learner 'imputemode.imputehist.encode.classif.rpart' on task 'tarjetas' (iter 7/10) 
INFO  [20:29:39.383] Applying learner 'imputemode.imputehist.encode.classif.rpart' on task 'tarjetas' (iter 3/10) 
INFO  [20:29:39.958] Applying learner 'imputemode.imputehist.encode.classif.rpart' on task 'tarjetas' (iter 2/10) 
INFO  [20:29:40.534] Applying learner 'imputemode.imputehist.encode.classif.rpart' on task 'tarjetas' (iter 6/10) 

confusion = rr$prediction()$confusion
print(confusion)

        truth
response bueno  malo
   bueno 13216   953
   malo    118   213

learner = GraphLearner$new(pre_procesamiento %>>% po(lrn("classif.rpart"))) 
rr = resample(task_tarj, learner, rsmp("cv"))

INFO  [20:29:41.599] Applying learner 'imputemode.imputehist.encode.classif.rpart' on task 'tarjetas' (iter 2/10) 
INFO  [20:29:42.633] Applying learner 'imputemode.imputehist.encode.classif.rpart' on task 'tarjetas' (iter 10/10) 
INFO  [20:29:43.362] Applying learner 'imputemode.imputehist.encode.classif.rpart' on task 'tarjetas' (iter 1/10) 
INFO  [20:29:44.141] Applying learner 'imputemode.imputehist.encode.classif.rpart' on task 'tarjetas' (iter 6/10) 
INFO  [20:29:44.889] Applying learner 'imputemode.imputehist.encode.classif.rpart' on task 'tarjetas' (iter 3/10) 
INFO  [20:29:45.633] Applying learner 'imputemode.imputehist.encode.classif.rpart' on task 'tarjetas' (iter 4/10) 
INFO  [20:29:46.370] Applying learner 'imputemode.imputehist.encode.classif.rpart' on task 'tarjetas' (iter 9/10) 
INFO  [20:29:47.769] Applying learner 'imputemode.imputehist.encode.classif.rpart' on task 'tarjetas' (iter 7/10) 
INFO  [20:29:48.503] Applying learner 'imputemode.imputehist.encode.classif.rpart' on task 'tarjetas' (iter 5/10) 
INFO  [20:29:49.245] Applying learner 'imputemode.imputehist.encode.classif.rpart' on task 'tarjetas' (iter 8/10) 

confusion = rr$prediction()$confusion
print(confusion)

        truth
response bueno  malo
   bueno 13214   956
   malo    120   210

avg_costs = sum(confusion * costs) / 1000
print(avg_costs)

[1] -0.069

-0.069*1000*1000

[1] -69000

cost_measure = msr("classif.costs", costs = costs)
print(cost_measure)

<MeasureClassifCosts:classif.costs>
* Packages: -
* Range: [-Inf, Inf]
* Minimize: TRUE
* Properties: requires_task
* Predict type: response

learners = list(
  GraphLearner$new(pre_procesamiento %>>% lrn("classif.log_reg")),
  GraphLearner$new(pre_procesamiento %>>% lrn("classif.featureless")),
  GraphLearner$new(pre_procesamiento %>>% lrn("classif.ranger")),
  GraphLearner$new(pre_procesamiento %>>% lrn("classif.rpart"))

)
cv3 = rsmp("cv", folds = 3)
bmr = benchmark(benchmark_grid(task_tarj, learners, cv3))

INFO  [20:45:11.389] Benchmark with 12 resampling iterations 
INFO  [20:45:11.411] Applying learner 'imputemode.imputehist.encode.classif.ranger' on task 'tarjetas' (iter 3/3) 
INFO  [20:45:17.241] Applying learner 'imputemode.imputehist.encode.classif.featureless' on task 'tarjetas' (iter 2/3) 
INFO  [20:45:17.773] Applying learner 'imputemode.imputehist.encode.classif.log_reg' on task 'tarjetas' (iter 3/3) 
INFO  [20:45:18.581] Applying learner 'imputemode.imputehist.encode.classif.rpart' on task 'tarjetas' (iter 1/3) 
INFO  [20:45:19.347] Applying learner 'imputemode.imputehist.encode.classif.log_reg' on task 'tarjetas' (iter 1/3) 
INFO  [20:45:20.916] Applying learner 'imputemode.imputehist.encode.classif.ranger' on task 'tarjetas' (iter 1/3) 
INFO  [20:45:26.569] Applying learner 'imputemode.imputehist.encode.classif.log_reg' on task 'tarjetas' (iter 2/3) 
INFO  [20:45:27.444] Applying learner 'imputemode.imputehist.encode.classif.featureless' on task 'tarjetas' (iter 3/3) 
INFO  [20:45:27.918] Applying learner 'imputemode.imputehist.encode.classif.ranger' on task 'tarjetas' (iter 2/3) 
INFO  [20:45:32.840] Applying learner 'imputemode.imputehist.encode.classif.rpart' on task 'tarjetas' (iter 3/3) 
INFO  [20:45:33.516] Applying learner 'imputemode.imputehist.encode.classif.featureless' on task 'tarjetas' (iter 1/3) 
INFO  [20:45:34.024] Applying learner 'imputemode.imputehist.encode.classif.rpart' on task 'tarjetas' (iter 2/3) 

prediction from a rank-deficient fit may be misleadingprediction from a rank-deficient fit may be misleadingprediction from a rank-deficient fit may be misleading

INFO  [20:45:34.677] Finished benchmark 

bmr$aggregate(cost_measure)

learners = list(
  GraphLearner$new(pre_procesamiento %>>% lrn("classif.log_reg")),
  GraphLearner$new(pre_procesamiento %>>% lrn("classif.featureless")),
  GraphLearner$new(pre_procesamiento %>>% lrn("classif.ranger")),
  GraphLearner$new(pre_procesamiento %>>% lrn("classif.rpart"))

)
cv3 = rsmp("cv", folds = 3)
bmr = benchmark(benchmark_grid(task_tarj, learners, cv3))

INFO  [20:45:37.120] Benchmark with 12 resampling iterations 
INFO  [20:45:37.139] Applying learner 'imputemode.imputehist.encode.classif.rpart' on task 'tarjetas' (iter 3/3) 
INFO  [20:45:38.403] Applying learner 'imputemode.imputehist.encode.classif.rpart' on task 'tarjetas' (iter 2/3) 
INFO  [20:45:39.019] Applying learner 'imputemode.imputehist.encode.classif.ranger' on task 'tarjetas' (iter 3/3) 
INFO  [20:45:43.722] Applying learner 'imputemode.imputehist.encode.classif.rpart' on task 'tarjetas' (iter 1/3) 
INFO  [20:45:44.384] Applying learner 'imputemode.imputehist.encode.classif.log_reg' on task 'tarjetas' (iter 2/3) 
INFO  [20:45:45.214] Applying learner 'imputemode.imputehist.encode.classif.featureless' on task 'tarjetas' (iter 1/3) 
INFO  [20:45:45.752] Applying learner 'imputemode.imputehist.encode.classif.featureless' on task 'tarjetas' (iter 2/3) 
INFO  [20:45:46.214] Applying learner 'imputemode.imputehist.encode.classif.featureless' on task 'tarjetas' (iter 3/3) 
INFO  [20:45:46.773] Applying learner 'imputemode.imputehist.encode.classif.ranger' on task 'tarjetas' (iter 1/3) 
INFO  [20:45:52.642] Applying learner 'imputemode.imputehist.encode.classif.log_reg' on task 'tarjetas' (iter 1/3) 
INFO  [20:45:53.583] Applying learner 'imputemode.imputehist.encode.classif.ranger' on task 'tarjetas' (iter 2/3) 
INFO  [20:45:59.230] Applying learner 'imputemode.imputehist.encode.classif.log_reg' on task 'tarjetas' (iter 3/3) 

prediction from a rank-deficient fit may be misleadingprediction from a rank-deficient fit may be misleadingprediction from a rank-deficient fit may be misleading

INFO  [20:46:00.062] Finished benchmark 

bmr$aggregate(cost_measure)

print(p)

<PredictionClassif> for 14500 observations:

print(p)

<PredictionClassif> for 14500 observations:

with_threshold = function(p, th) {
  p$set_threshold(th)
  list(confusion = p$confusion, costs = p$score(measures = cost_measure, task = task_tarj))
}

with_threshold(p, 0.5)

$confusion
        truth
response bueno  malo
   bueno 13334  1166
   malo      0     0

$costs
classif.costs 
            0 

with_threshold(p, 0.5)

$confusion
        truth
response bueno  malo
   bueno 13334  1166
   malo      0     0

$costs
classif.costs 
            0 

with_threshold(p, 0.75)

$confusion
        truth
response bueno  malo
   bueno 13334  1166
   malo      0     0

$costs
classif.costs 
            0 

with_threshold(p, 0.92)

$confusion
        truth
response bueno malo
   bueno  7461  515
   malo   5873  651

$costs
classif.costs 
    0.3646276 

print(costs)

        truth
response bueno malo
   bueno     0  0.0
   malo      1 -0.5